import pandas as pd
import os
import sys


"""

注意事项：
base_dir 需要和get_all_real_total_time_and_predict_time_to_summary_csv.py中的地址保持一致

执行时机:
执行c-lop_analyze.py 之后,执行get_avg_precision_from_summary_csv.py 之后，想要获取平均误差


该脚本的作用：
会将'cn_dir','node_dir', 'proc_dir', 'comm_type'相同的数据进行分组，并计算平均误差

"""

def process_summary_csv(file_path):
    try:
        # 正确读取CSV文件，使用第一行作为标题
        df = pd.read_csv(file_path, quotechar='"', skipinitialspace=True)
        
        print(f"原始数据行数: {len(df)}")
        print(f"列名: {df.columns.tolist()}")
        
        # 清洗数据 - 移除空行或部分空行
        df = df.dropna(how='all')
        
        # 确保数值列是数值类型
        df['total_real_comm_time'] = pd.to_numeric(df['total_real_comm_time'], errors='coerce')
        df['predict_total_comm_time'] = pd.to_numeric(df['predict_total_comm_time'], errors='coerce')
        df['|total_real_comm_time - predict_total_comm_time| / total_real_comm_time'] = pd.to_numeric(df['|total_real_comm_time - predict_total_comm_time| / total_real_comm_time'], errors='coerce')
        
        # 过滤掉有缺失值的行
        df = df.dropna(subset=['total_real_comm_time', 'predict_total_comm_time', '|total_real_comm_time - predict_total_comm_time| / total_real_comm_time'])
        
        print(f"清洗后数据行数: {len(df)}")
        print(f"不同node_dir的值: {df['node_dir'].unique()}")
        
        # 按指定字段分组并计算平均值
        grouped = df.groupby(['node_dir', 'proc_dir', 'comm_type'])
        result = grouped.agg({
            'total_real_comm_time': 'mean',
            'predict_total_comm_time': 'mean',
            '|total_real_comm_time - predict_total_comm_time| / total_real_comm_time': 'mean'
        }).reset_index()
        
        # 重命名结果列为更有意义的名称
        result.rename(columns={
            'total_real_comm_time': 'avg_total_real_comm_time', 
            'predict_total_comm_time': 'avg_predict_total_comm_time',
            '|total_real_comm_time - predict_total_comm_time| / total_real_comm_time': 'avg_relative_error'
        }, inplace=True)
        
        print(f"分组后结果行数: {len(result)}")
        print(f"结果中的node_dir值: {result['node_dir'].unique()}")
        
        return result
        
    except Exception as e:
        print(f"处理CSV文件时出错: {e}")
        # 打印前几行数据帮助调试
        if 'df' in locals():
            print("\n调试用样本数据:")
            print(df.head(3))
            print("\n数据类型:")
            print(df.dtypes)
        raise

if __name__ == "__main__":
    # 使用原始字符串处理Windows路径
    csv_dir = r"F:\PostGraduate\Point-to-Point-Code\App_Prediction\analysis_weak_scaling\analysis_for_100atom_per_proc\all_predict_precision"


    
    # 使用os.path.join确保路径正确
    csv_file = os.path.join(csv_dir, "summary.csv")
    avg_precision_csv_dir = os.path.join(csv_dir, "avg_precision_summary.csv")
    
    if not os.path.exists(csv_file):
        print(f"Error: File not found: {csv_file}")
        sys.exit(1)
    
    try:
        # 处理CSV文件并输出结果
        result_df = process_summary_csv(csv_file)
        
        # 保存结果前确保没有空值
        result_df = result_df.dropna()
        
        # 保存结果
        result_df.to_csv(avg_precision_csv_dir, index=False)
        print(f"Successfully saved results to: {avg_precision_csv_dir}")
        
        # 添加验证：读取并显示保存的文件最后几行
        print("\nResults file preview (last 5 rows):")
        preview_df = pd.read_csv(avg_precision_csv_dir)
        print(preview_df.tail(5))
    except Exception as e:
        print(f"Error occurred: {e}")
        sys.exit(1)